package com.yangyang.lamviarm.client;

import java.awt.JobAttributes;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Random;

import javax.sql.rowset.Predicate;

import com.yangyang.lamviarm.entity.ErrorActionModelException;
import com.yangyang.lamviarm.entity.JActionDef;
import com.yangyang.lamviarm.entity.JActionModelDef;
import com.yangyang.lamviarm.entity.JPredicate;
import com.yangyang.lamviarm.entity.JPredicateDef;
import com.yangyang.lamviarm.server.JServer;
import com.yangyang.lamviarm.server.JSimulator;
import com.yangyang.lamviarm.util.ClientActionEvent;
import com.yangyang.lamviarm.util.ClientActionListener;
import com.yangyang.lamviarm.util.ClientTerminateEvent;
import com.yangyang.lamviarm.util.Constants;
import com.yangyang.lamviarm.util.TypeEnum;

/*
 * learn action-model via reinforcement learning
 */
public class LAMVaiRL {

	private HashMap<String, List<JActionModelDef>> mActionTable;
	private List<JPredicate> currentState;
	private List<JPredicate> potentialState; // Ǳ����ȷ��״̬
	private List<JPredicate> goalState;
	private String mNextAction;
	private JActionModelDef mNextActionModel;
	private ClientActionListener listener;

	public LAMVaiRL() {

		this.mActionTable = new HashMap<String, List<JActionModelDef>>();
		this.currentState = new ArrayList<JPredicate>();
		this.potentialState = new ArrayList<JPredicate>();
		this.goalState = new ArrayList<JPredicate>();

		// enum all possible action-model
		for (JActionDef actionDef : JActionDef.mapNameActionDef.values()) {

			// 获取动作名称和参数列表
			List<String> listAction = new ArrayList<String>();
			HashMap<TypeEnum, Integer> typeNumber = new HashMap<TypeEnum, Integer>();
			listAction.add(actionDef.getName());
			for (String s : actionDef.getParamName())
				listAction.add(s);
			for (TypeEnum type : actionDef.getParamType()) {
				if (typeNumber.containsKey(type))
					typeNumber.put(type, typeNumber.get(type) + 1);
				else
					typeNumber.put(type, 1);
			}

			// 获取所有潜在的谓词定义
			List<JPredicateDef> avalibleDef = new ArrayList<JPredicateDef>();
			for (JPredicateDef def : JPredicateDef.mapNamePredicate.values()) {
				boolean flag = true;
				for (TypeEnum type : def.getParamType()) {
					if ((!typeNumber.containsKey(type) && type != TypeEnum.OBJECT)
							|| (!typeNumber.containsKey(TypeEnum.ROBOT) && type == TypeEnum.OBJECT)) {
						flag = false;
						break;
					}
				}
				if (flag)
					avalibleDef.add(def);
			}

			// 枚举谓词使其符合typenumber中定义的参数类型和个数
			List<List<String>> possibleDef = new ArrayList<List<String>>();
			for (JPredicateDef def : avalibleDef) {
				if (def.getParamType() == null
						|| def.getParamType().length == 0) { // 没有参数的谓词肯定符合要求啦
					List<String> newpre = new ArrayList<String>();
					newpre.add(def.getName());
					possibleDef.add(newpre);
				}
			}
			possibleDef.add(new ArrayList<String>());

			for (int i = 1; i < actionDef.getParamType().length; i++) {
				for (JPredicateDef def : avalibleDef) { // 谓词参数个数为1
					TypeEnum[] paramType = def.getParamType();
					if (paramType.length != 1)
						continue;
					for (int p = 0; p < paramType.length; p++) {
						if (paramType[p] == actionDef.getParamType()[i]) {
							List<List<String>> tempDef = new ArrayList<List<String>>();
							for (List<String> modelDef : possibleDef) {
								List<String> newModelDef = new ArrayList<String>();
								newModelDef.addAll(modelDef);
								newModelDef.add(def.getName() + "_"
										+ actionDef.getParamName()[i]);
								tempDef.add(newModelDef);
							}
							possibleDef.addAll(tempDef);
							break;
						}
					}
				}
			}

			for (int i = 0; i < actionDef.getParamType().length; i++) {
				for (int j = 0; j < actionDef.getParamType().length; j++) {
					if (i == j)
						continue;
					List<TypeEnum> actionType = new ArrayList<TypeEnum>();
					actionType.add(actionDef.getParamType()[i]);
					actionType.add(actionDef.getParamType()[j]);
					for (JPredicateDef def : avalibleDef) { // 谓词参数个数为2
						TypeEnum[] paramType = def.getParamType();
						if (paramType.length != 2)
							continue;
						if (TypeEnum.exactFit(actionType, paramType)) {
							List<List<String>> tempDef = new ArrayList<List<String>>();
							for (List<String> modelDef : possibleDef) {
								List<String> newModelDef = new ArrayList<String>();
								newModelDef.addAll(modelDef);
								newModelDef.add(def.getName() + "_"
										+ actionDef.getParamName()[i] + "_"
										+ actionDef.getParamName()[j]);
								tempDef.add(newModelDef);
							}
							possibleDef.addAll(tempDef);
							break;
						}
					}
				}
			}
			this.GeneratePossibleActionModel(actionDef.getName(), possibleDef);
		}

		this.readTrainData();
		// System.out.println("size:" + this.mActionTable.keySet().size());
		// for (String action : this.mActionTable.keySet()) {
		// System.out.println("\n\n action table :" + action + "\n");
		// List<JActionModelDef> models = this.mActionTable.get(action);
		// System.out.println(models);
		// System.out.println("\n");
		// }
	}

	/**
	 * 读取初始状态和目标状态
	 */
	public void readTrainData() {
		this.potentialState.clear();
		this.currentState.clear();
		for (JPredicate pre : JSimulator.currentstate) {
			this.currentState.add(pre);
		}

		this.goalState.clear();
		for (JPredicate pre : JSimulator.goalstate) {
			this.goalState.add(pre);
		}

	}

	/**
	 * 找到每一个anctionmoel的中位数，并把Q值在中位数之上的去掉
	 */
	public void clearActionModel() {
		for (String action : this.mActionTable.keySet()) {
			List<JActionModelDef> list = this.mActionTable.get(action);
			if (list.size() < 10)
				continue;
			List<JActionModelDef> toRemove = new ArrayList<JActionModelDef>();
			List<Integer> values = new ArrayList<Integer>();
			for (JActionModelDef model : list) {
				if (!values.contains(model.getQValue())) {
					values.add(model.getQValue());
				}
				model.mQValueList.clear();
			}
			if (true || values.size() < 5)
				return;
			Collections.sort(values);
			int Median = values.get(values.size() - 2);
			for (JActionModelDef model : list) {
				if (model.getQValue() > Median)
					toRemove.add(model);
			}
			list.removeAll(toRemove);

		}
	}

	private void GeneratePossibleActionModel(String modelName,
			List<List<String>> possibleModels) {
		// 必须要参数全部满足
		for (List<String> lines : possibleModels) {
			if (!JActionDef.mapNameActionDef.get(modelName).fit(lines))
				continue;
			else {
				List<JActionModelDef> models = new ArrayList<JActionModelDef>();
				models.add(new JActionModelDef(modelName,
						JActionDef.mapNameActionDef.get(modelName)
								.getParamName()));
				for (String pre : lines) {
					List<JActionModelDef> tempModels = new ArrayList<JActionModelDef>();
					for (JActionModelDef model : models) {
						if (model.bePrecondition(pre)) {
							JActionModelDef newModel = model.clone();
							newModel.mPrecondition.add(pre);
							tempModels.add(newModel);
						}
						if (JPredicate.canBeEffect(pre) && model.beEffectA(pre)) {
							JActionModelDef newModel = model.clone();
							newModel.mEffectA.add(pre); // pre加入effectA，必须把与pre矛盾的添加到EffectD中
							List<String> toAddEffectD = new ArrayList<String>();
							for (String precondition : newModel.mPrecondition) {
								if (JPredicate.contraPredicate(precondition,
										pre) && newModel.bEffectD(precondition))
									toAddEffectD.add(precondition);
							}
							newModel.mEffectD.addAll(toAddEffectD);
							tempModels.add(newModel);
						}
					}
					models = tempModels;
				}

				for (JActionModelDef model : models) {
					List<JActionModelDef> tempModels = new ArrayList<JActionModelDef>();
					tempModels.add(model);
					for (String pre : model.mPrecondition) { // 把所有可以做effectd的谓词都遍历一遍
						List<JActionModelDef> newModels = new ArrayList<JActionModelDef>();
						for (JActionModelDef tempmodel : tempModels) {
							if (JPredicate.canBeEffect(pre)
									&& tempmodel.bEffectD(pre)) {
								JActionModelDef newmodel = tempmodel.clone();
								newmodel.mEffectD.add(pre);
								newModels.add(newmodel);
							}
						}
						tempModels.addAll(newModels);
					}
					// System.out.println("==============");
					// System.out.println(tempModels);
					// System.out.println("==============");
					for (JActionModelDef actionModel : tempModels) {
						if (actionModel.mPrecondition.size() > 0
								&& actionModel.mEffectA.size() > 0
								&& actionModel.mEffectD.size() > 0) {
							List<JActionModelDef> actionModels = this.mActionTable
									.get(actionModel.getName());
							if (actionModels == null) {
								actionModels = new ArrayList<JActionModelDef>();
								this.mActionTable.put(actionModel.getName(),
										actionModels);
							}
							actionModels.add(actionModel);
						}
					}
				}

			}
		}
	}

	public void setClientActionListener(ClientActionListener listener) {
		this.listener = listener;
	}

	public HashMap<String, List<JActionModelDef>> getActionTable() {
		return this.mActionTable;
	}

	/**
	 * check whether the iterator can be stopped. condition: reached goal or q
	 * value is no longer changed.
	 * 
	 * @return
	 */
	private boolean canFinish() {
		boolean result = true;
		// already reached the goal
		for (JPredicate pre : this.goalState) {
			if (!this.currentState.contains(pre)) {
				result = false;
				break;
			}
		}

		if (!result) {
			result = true;
			for (String key : this.mActionTable.keySet()) {
				List<JActionModelDef> models = this.mActionTable.get(key);
				JActionModelDef model = models.get(0);
				for (int i = 1; i < models.size(); i++) {
					if (models.get(i).getQValue() < model.getQValue()) {
						model = models.get(i);
					}
				}
				if (model.getQValue() <= 0
						|| model.getQValue() > 0
						&& Math.abs(model.getQValue() - model.getOldQValue()) > 5) {
					result = false;
					break;
				}
			}
		}
		return result;
	}

	private boolean nextAction() {
		this.mNextAction = null;
		boolean result = true;
		int currentValue = Constants.JMAX + 1;
		boolean loop = true;
		boolean[] actionFlag = new boolean[this.mActionTable.size()];
		for (int i = 0; i < actionFlag.length; i++)
			actionFlag[i] = false;
		while (loop) {
			loop = false;
			int Index = new Random().nextInt(this.mActionTable.keySet().size());

			if (actionFlag[Index])
				continue;
			{
				String action = this.mActionTable.keySet().toArray()[Index]
						.toString();
				List<JActionModelDef> models = this.mActionTable.get(action);
				assert (models != null && models.size() > 0);
				JActionModelDef model = models.get(0);
				for (int i = 1; i < models.size(); i++) {
					if (models.get(i).getQValue() < model.getQValue()
							&& (models.get(i).getQValue() == model.getQValue() && new Random()
									.nextInt(10) > 5))
						model = models.get(i);
				}

				// 看当前的actionmodel能得多少分
				List<String> actionvalue = null;
				try {
					actionvalue = this.enumActionValues(model);
				} catch (ErrorActionModelException e) {
					// TODO Auto-generated catch block
					loop = true;
					models.remove(model);
					continue;
				}
				if (actionvalue.size() == 0) {
					loop = true;
					continue;
				} else {
					actionFlag[Index] = true;
				}
				for (String line : actionvalue) {
					int v = this.prepareAction(model, line);
					if (v < currentValue
							|| (v == currentValue && new Random().nextInt(10) > 5)) {
						currentValue = v;
						this.mNextAction = line;
						this.mNextActionModel = model;
					}
					if (v == Constants.JMAX)
						result = false;
				}
			}

			boolean com = true;
			for (int i = 0; i < actionFlag.length; i++) {
				if (!actionFlag[i]) {
					com = false;
					break;
				}
			}

			if (!com && new Random().nextBoolean()) {
				loop = true;
			}
		}

		return result && this.mNextAction != null;
	}

	private List<String> enumActionValues(JActionModelDef model)
			throws ErrorActionModelException {
		HashMap<TypeEnum, List<String>> typeInstance = new HashMap<TypeEnum, List<String>>();
		for (JPredicate pre : this.currentState) {
			JPredicateDef def = JPredicateDef.mapNamePredicate.get(pre
					.getName());
			for (int i = 0; i < def.getParamType().length; i++) {
				List<String> instance = typeInstance.get(def.getParamType()[i]);
				if (instance == null) {
					instance = new ArrayList<String>();
					typeInstance.put(def.getParamType()[i], instance);
				}
				if (!instance.contains(pre.getParams()[i].toString())
						&& !pre.getParams()[i].toString().equals("r1"))
					instance.add(pre.getParams()[i].toString());
			}
		}

		JActionDef action = JActionDef.mapNameActionDef.get(model.getName());
		String actInstance = action.getName() + "_r1";
		List<String> actions = new ArrayList<String>();
		actions.add(actInstance);
		for (int i = 0; i < action.getParamType().length; i++) {
			if (action.getParamType()[i] == TypeEnum.ROBOT)
				continue;
			List<String> instance = typeInstance.get(action.getParamType()[i]);
			assert (instance.size() != 0);

			List<String> temp = new ArrayList<String>();
			for (String str : actions) {
				for (String var : instance) {
					temp.add(str + "_" + var);
				}

			}
			actions = temp;
			if (i == action.getParamType().length - 1) {
			}
		}
		List<String> result = new ArrayList<String>();
		for (String line : actions) {
			boolean theoryRight = JPredicate.actionCorrent(line, model,
					this.currentState);
			boolean actualRight = JSimulator.simulator.rightAction(line,
					new HashMap<String, String>());
			if (actualRight) {
				result.add(line);
			} else {
				if (theoryRight) {
					throw new ErrorActionModelException();
				}
				if (new Random().nextInt(100) < 2) {
					result.add(line);
				}
			}
		}
		return result;
	}

	// 根据当前的action预估执行actionmodel后可以获得的结果（与目标状态的距离）
	private int prepareAction(JActionModelDef model, String action) {
		int result = 0;
		List<JPredicate> states = new ArrayList<JPredicate>();
		List<JPredicate> NewStates = new ArrayList<JPredicate>();
		List<JPredicate> RemoveStates = new ArrayList<JPredicate>();
		for (JPredicate pre : this.currentState)
			states.add(pre);

		// List<String> variable = this.mNextActionModel.mActionVariable; --fix
		// bug yangyang 20111130
		List<String> variable = model.mActionVariable;
		HashMap<String, String> mapVariable = new HashMap<String, String>();
		// String[] array = this.mNextAction.split("_");//fixbug yangyang
		// 20111130
		String[] array = action.split("_");
		for (int i = 0; i < variable.size(); i++) {
			mapVariable.put(variable.get(i), array[i + 1]);
		}
		for (String str : model.mPrecondition) { // 检查是否满足当前条件
			if (!JPredicate.predicateCorrect(str, mapVariable, states))
				return Constants.JMAX;
		}

		// 判断执行后的预计状态与目标状态的距离
		for (String line : model.mEffectA) {
			JPredicate pre = JPredicate.parseByVar(line, mapVariable, states);
			if (pre == null) {
				System.err.println("in prepareAction :" + pre);
			} else {
				if (!states.contains(pre)) {
					states.add(pre);
					NewStates.add(pre);
				}
			}
		}

		for (String line : model.mEffectD) {
			JPredicate pre = JPredicate.parseByVar(line, mapVariable, states);
			if (pre == null) {
				System.err.println("in prepareAction :" + pre);
			} else {
				if (states.contains(pre)) {
					states.remove(pre);
					RemoveStates.add(pre);
				}
			}
		}

		for (JPredicate pre : this.goalState) {
			if (!states.contains(pre)) {
				JPredicate p = null;
				if (pre.getName().equals("location")) {
					for (JPredicate jp : states) {
						if (jp.getName().equals("location")
								&& jp.getParams()[0].equals(pre.getParams()[0])) {
							p = jp;
							break;
						} else if (jp.getName().equals("holding")
								&& jp.getParams()[0].equals(pre.getParams()[0])) { // holding的话取机器人的位置
							for (JPredicate jp2 : states) {
								if (jp2.getName().equals("location")
										&& jp2.getParams()[0].equals("r1")) {
									p = jp2;
									break;
								}
							}
							break;
						}
					}
				}
				if (p == null)
					result += 100;
				else {
					int l1 = Integer.parseInt(p.getParams()[1].toString());
					int l2 = Integer.parseInt(pre.getParams()[1].toString());
					result += Math.abs(l1 / 1000 - l2 / 1000)
							+ Math.abs(l1 % 1000 - l2 % 1000);
				}
			}
		}

		return result;
	}

	public void run() {
		final int maxIter = 5000;
		int iter = 0;
		for (; iter < maxIter; iter++) {
			if (canFinish()) {
				System.out.println("---------- finish(canfinish) ----------");
				if (this.listener != null)
					this.listener.OnClientTeminated(new ClientTerminateEvent(
							this, 1));
				break;
			} else if (iter == maxIter - 1 && this.listener != null) {
				System.out.println("---------- finish(maxiter) ----------");
				this.listener.OnClientTeminated(new ClientTerminateEvent(this,
						2));
			}

			boolean fit = nextAction();// 当前的环境是否满足下一个动作
			String action = this.mNextAction;
			if (action == null) { // 找不到下一个anction了
				System.out
						.println("---------- finish(nextaction null) ----------");
				if (this.listener != null)
					this.listener.OnClientTeminated(new ClientTerminateEvent(
							this, 3));
				break;
			}
			String[] array = action.split("_");
			List<JActionModelDef> models = this.mActionTable.get(array[0]);
			assert (models != null && models.size() > 0);
			JActionModelDef model = models.get(0);
			int maxQValue = model.getQValue();
			for (int i = 1; i < models.size(); i++) {
				if (models.get(i).getQValue() < model.getQValue())
					model = models.get(i);
				if (models.get(i).getQValue() > maxQValue)
					maxQValue = models.get(i).getQValue();
			}

			List<JPredicate> status = new ArrayList<JPredicate>();
			boolean result = JSocketClient.doAction(action, status);
			if (!result) {
				if (fit) {
					models.remove(model);
				} else {
					model.setQValue(maxQValue + 10); // ��model��Qalue������
				}
			} else { // �����ɹ������µ�ǰ״̬��Qֵ�ȡ�
				this.doAction(model, action, status);
			}

			if (this.listener != null) {
				listener.OnClientActionDone(new ClientActionEvent(this,
						this.mActionTable, "action:" + action + "\tresult:"
								+ result));
				// try {
				// Thread.sleep(2000);
				// } catch (InterruptedException e) {
				// // TODO Auto-generated catch block
				// e.printStackTrace();
				// }
			}

		}

		for (String name : this.mActionTable.keySet()) {
			JServer.writer.println("action-model " + name + " number:"
					+ this.mActionTable.get(name).size());
			int minQ = Constants.JMAX;
			for (JActionModelDef def : this.mActionTable.get(name)) {
				JServer.writer.println(def.toString() + "==="
						+ def.toQListString());
				if (def.getQValue() < minQ)
					minQ = def.getQValue();
			}
			JServer.writer.println("min q value:" + minQ);
		}
		JServer.writer.println("iter time:" + iter);
	}

	/**
	 * ���ѡȡ��actionmodel�����������ѽ���ʵ�ʵĽ��Աȣ�����Qֵ��
	 * 
	 * @param model
	 * @param action
	 * @param realResult
	 */
	private void doAction(JActionModelDef model, String action,
			List<JPredicate> status) {

		List<JPredicate> RealPre = new ArrayList<JPredicate>();
		List<JPredicate> EfffectA = new ArrayList<JPredicate>();
		List<JPredicate> EfffectD = new ArrayList<JPredicate>();

		for (JPredicate pre : status) {
			assert (pre != null);
			RealPre.add(pre);
			for (int i = this.currentState.size() - 1; i >= 0; i--) {
				if (this.currentState.get(i).contraPredicate(pre)) { // ��ǰ��״̬���Ѿ����ڵ�״̬�г�ͻ����ɾ���Ѿ����ڵ�״̬
					this.currentState.remove(i);
				}
			}

			for (int i = this.potentialState.size() - 1; i >= 0; i--) {
				if (this.potentialState.get(i).contraPredicate(pre)) { // Ǳ�ڵ�״̬���Ѿ����ڵ�״̬�г�ͻ����ɾ���Ѿ����ڵ�״̬
					this.potentialState.remove(i);
				}
			}
			if (!this.currentState.contains(pre))
				this.currentState.add(pre);
		}

		// ���Ǳ�ڵĶ���
		HashMap<String, String> mapVariableValue = new HashMap<String, String>();
		String[] array = action.split("_");
		assert (array.length == model.mActionVariable.size() + 1);
		for (int i = 0; i < model.mActionVariable.size(); i++) {
			mapVariableValue.put(model.mActionVariable.get(i), array[i + 1]);
		}

		for (String predicateStr : model.mEffectA) {

			EfffectA.add(JPredicate.parseByVar(predicateStr, mapVariableValue,
					goalState));
		}

		for (String predicateStr : model.mEffectD) {
			EfffectD.add(JPredicate.parseByVar(predicateStr, mapVariableValue,
					goalState));
		}

		// 更新model的QValue
		int qValue = 1;
		for (JPredicate pre1 : RealPre) {
			if (qValue >= Constants.JMAX)
				break;
			if (EfffectA.contains(pre1)) // 包含pre，则不惩罚
				continue;
			if (pre1.containContraPredicate(EfffectA)
					|| EfffectD.contains(pre1)) // 包含相反的谓词,给最大的惩罚
			{
				qValue += Constants.JMAX;
				break;
			}

			boolean flag = false;
			for (JPredicate pre2 : EfffectA) {
				if (pre1.getName().equals(pre2.getName())) {
					qValue += JPredicate.getPreDistance(pre1, pre2);
					flag = true;
				}
			}
			if (!flag) // 实际的pre在理论上的actionmodel没有体现
				qValue += 100;
		}

		if (qValue < Constants.JMAX) {
			for (JPredicate pre : EfffectA) {
				if (!currentState.contains(pre)) // 理论上的pre在实际上没有观察到
					qValue += 10;
			}
		}

		if (qValue < Constants.JMAX) {
			model.setQValue(qValue);
		} else {
			List<JActionModelDef> models = this.mActionTable.get(model
					.getName());
			models.remove(model);
		}
	}

	@Override
	public String toString() {
		StringBuilder sb = new StringBuilder();
		if (this.mActionTable != null) {
			for (String key : this.mActionTable.keySet()) {
				List<JActionModelDef> models = this.mActionTable.get(key);
				JActionModelDef model = models.get(0);
				for (int i = 1; i < models.size(); i++) {
					if (models.get(i).getQValue() < model.getQValue())
						model = models.get(i);
				}

				sb.append("Action : " + key + "\n");
				sb.append("\tprecondition:");
				for (String str : model.mPrecondition)
					sb.append(str + " ");
				sb.append("\n");

				sb.append("\teffecta:");
				for (String str : model.mEffectA)
					sb.append(str + " ");
				sb.append("\n");

				sb.append("\teffectd:");
				for (String str : model.mEffectD)
					sb.append(str + " ");
				sb.append("\n");
			}
		}
		return sb.toString();
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
	}

}
